#! /usr/bin/python

#### Parallel rsync handler ####
## The goal of this program is to sync a base dir and all subdirs with a target location.
## It will treat each subdir as a seperate 'job' with 'n' threads being run at a time.  Currently
## has no support for config files or SSH but may be added in the future via a credential file setup.
## It would however presumably work if you were using a passcode free sshkey

# This program is free software; you can redistribute it and/or
# modify it under the terms of the GNU General Public License
# as published by the Free Software Foundation; either version 2
# of the License, or (at your option) any later version.

# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU General Public License for more details.

# You should have received a copy of the GNU General Public License
# along with this program; if not, write to the Free Software
# Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.


# System modules
import Queue
import threading
from optparse import OptionParser #ideally this should be argparse but that isn't available in 2.6.6
#import time
import os
#from sys import exit, exc_info
import sys
import subprocess
import logging

class rsyncWorker(threading.Thread):
    def __init__(self, srcdir, destdir, subdirQ):
        threading.Thread.__init__(self)
        self.srcdir = srcdir
        self.destdir = destdir
        self.subdirQ = subdirQ
        self.continueProcess = True



    @classmethod
    def kill(self):
        self.continueProcess = False

    def run(self):
    #execution steps.
        logging.log(19,(self.name + " Starting"))

        #self.continueProcess = True

        rsyncProcess = None

        while (not self.subdirQ.empty()) and self.continueProcess:

            #logging.debug(self.name + " Continue Process set to: " + str(self.continueProcess) + " at start of try/catch")

            try:

                logging.debug(self.name + ": subdirQ has " + str(self.subdirQ.qsize()) + " items")

                subdir = self.subdirQ.get_nowait()



                rsyncCommand = ""

                if subdir == "": #executing root rsync
                    logging.info((self.name + " handling sync of root folder"))
                    self.rsyncCommand = 'rsync -v -d \"' + self.srcdir + '/\" \"' + self.destdir + '\"'
                    logging.log(17, (self.name + ": RsyncCommand - " + rsyncCommand))

                else: #execute on local
                    logging.info((self.name + " handling sync of subdir " + subdir))
                    rsyncCommand = 'rsync -v -r \"' + self.srcdir + '/' + subdir + '\" \"'  + self.destdir + '\"'
                    logging.log(17, (self.name + ": RsyncCommand - " + rsyncCommand))

                rsyncProcess = subprocess.Popen(rsyncCommand, shell=True, stdout=subprocess.PIPE, stderr=subprocess.PIPE)


                while (rsyncProcess.poll() == None):

                    logging.debug(self.name + " polling on " + self.srcdir + '/' + subdir + " task")

                    if not self.continueProcess:
                        logging.error(self.name + " Killing Rsync due to Shutdown Signal")
                        rsyncProcess.kill()
                        rsyncProcess = None
                        logging.error(self.name + " Terminating Early via Force Quit")
                        break


                    outlines,errlines = rsyncProcess.communicate()

                    if errlines:
                        errlines = errlines.split('\n')
                        for line in errlines:
                            logging.error(self.name + ": " + str(line))

                    if outlines:
                        outlines = outlines.split('\n')
                        for line in outlines:
                            logging.info((self.name + ": " + str(line)))



                logging.info((self.name + " work complete on: " + self.srcdir + "/" + subdir))

                rsyncProcess = None #clear variable for next execution
                self.subdirQ.task_done()

            except Exception, ex:

                if (rsyncProcess!=None):
                    rsyncProcess.terminate()
                    rsyncProcess = None
                    logging.warn(self.name + " rsync process killed")

                logging.error((self.name + " Encountered Exception and is Quiting"))

                logging.error((self.name + ": " + repr(ex))) #use repr to dump the non pretty but more detailed technical string

                return 0



        logging.log(19,(self.name + " Work finished. Quiting."))
        return 0



def setupOptions():

    usage = "usage: %prog [options] SRC DEST"

    epilog = "SRC and DEST should be directories without trailing slashes (\) "
    parser = OptionParser(usage=usage, version="%prog 1.0", epilog=epilog)

    #add debug and verbosity
    parser.add_option("-d", "--debug", action="store_true", dest="debug", help="Turn on Debugging (all verbosity info + extra info)", default=False)
    parser.add_option("-v", "--verbose", action="count", dest="verbose", help="Increase output verbosity (1-3)", default=0)

    #add control for number of threads to execute
    parser.add_option("-n", "--nthreads", dest="nthreads", action="store", default=1, help="number of distinct rsync threads you want running at a time", metavar="NUMBER")

    (opts,args) = parser.parse_args()


    if opts.nthreads < 1:
        parser.error(" Can't have less than one thread ")

    if (opts.debug):

        logger = logging.getLogger() #retrieve root logger
        logger.setLevel(logging.DEBUG) #set output to debug
        logger.info("Logging set to Debug: " + str( logger.getEffectiveLevel() ) )

    elif (opts.verbosity):
        #logging.setLevel(verbosity)
        logger = logging.getLogger() #retrieve root logger
        logger.setLevel(logging.INFO - opts.verbosity) #set output to current verbosity level, lower numbers are more verbose
        logger.info("Logging set to Verbosity: " + str( logger.getEffectiveLevel() ) )

    logging.debug("Options parser built and executed")

    logging.log(19,("options", opts))
    logging.log(19,("args", args))

    if len(args) != 2:
        parser.error("Incorrect Number of Arguements")

    return (opts, args)


def getSubdirList(dir):
    logging.log(19, "Beginning getSubDirList Function")

    return [name for name in os.listdir(dir) if os.path.isdir(os.path.join(dir,name))] #returns a string for each subdir detected


def main():

    logging.basicConfig(level=logging.INFO)
    logging.info("Logging begun")


    opts,args = setupOptions()

    srcdir = args[0]
    destdir = args[1]
    #nthreads = opts.nthreads

    #setup data and queue
    subdirs = getSubdirList(srcdir)
    logging.log(19,"SubDirs: " + str(subdirs) )

    #prime the queue
    logging.log(18, "priming queue")

    subdirQ = Queue.Queue()

    subdirQ.put('') # we want this one to catch the root files

    logging.log(18, "loading subdirs into queue")

    for subdir in subdirs:
        subdirQ.put(subdir) #insert each subdir into the queue

    logging.log(17,("Queue Size: ", subdirQ.qsize()))

    threadPool = []


#wrap this in a try/catch for keyboard interrupt.

    for i in range(int(opts.nthreads)):
        logging.debug("Thread " + str(i) + " added to pool")
        threadPool.append(rsyncWorker(srcdir,destdir,subdirQ))
        threadPool[i].name = "Thread " + str(i)

    try:

        logging.debug("main thread try/catch entered")

        logging.log(18, "launching processor threads")


        for thread in threadPool:
            logging.debug(thread.name + " Launching")
            thread.start()

        logging.log(18, "Waiting for threads to complete work")

        #subdirQ.join()
        while not subdirQ.empty():
            pass



        for thread in threadPool:
            thread.join()

        subdirQ.join()

    except Exception as ex:

        logging.error("Exception caught.  Force Quitting")

        logging.error("Exception: " + repr(ex))


        if threadPool != []:
            for rsyncThread in threadPool:
                logging.error("Killing " + rsyncThread.name)
                rsyncThread.kill()
                rsyncThread.join()


        logging.error("Ending logging due to exception")
        logging.shutdown()

        return 0

## end try catch area.



    #treat queue as processing queue and wait till it's thread quits


    #subdirQ.join()

    logging.log(18,"Closing Program.")

    logging.shutdown()

    #teriminate program
    return 0



if __name__ == '__main__':
    exit(main())


# Copyright (c) <2011> <Garrett McGrath>

 # Permission is hereby granted, free of charge, to any person
 # obtaining a copy of this software and associated documentation
 # files (the "Software"), to deal in the Software without
 # restriction, including without limitation the rights to use,
 # copy, modify, merge, publish, distribute, sublicense, and/or sell
 # copies of the Software, and to permit persons to whom the
 # Software is furnished to do so, subject to the following
 # conditions:

 # The above copyright notice and this permission notice shall be
 # included in all copies or substantial portions of the Software.

 # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
 # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES
 # OF MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
 # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT
 # HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
 # WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING
 # FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR
 # OTHER DEALINGS IN THE SOFTWARE.
